#_*_coding:utf-8_*_ 

import requests
import time
import json
import re
import pymongo
from lxml import etree

header = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Cookie": "UM_distinctid=1773e923249593-018e6758a55b39-326a7006-116d20-1773e92324a88d; taihe_bi_sdk_uid=79ebdef05b9148a04f20fc8dcab1fd37; taihe_bi_sdk_session=03833765bf2bff57c7eee58ac77a0831; ngacn0comUserInfo=kkkboking%09kkkboking%0939%0939%09%0910%090%094%090%090%09; ngaPassportUid=62881991; ngaPassportUrlencodedUname=kkkboking; ngaPassportCid=X93ruib6jkbp6a5uok4df470l7drab3emjg4slsi; ngacn0comUserInfoCheck=0aae10da5b17736ebd18e986fdbf730e; ngacn0comInfoCheckTime=1614584924; CNZZDATA30043604=cnzz_eid%3D413352232-1611659283-https%253A%252F%252Fbbs.nga.cn%252F%26ntime%3D1614652870; bbsmisccookies=%7B%22uisetting%22%3A%7B0%3A%22c%22%2C1%3A1614653734%7D%2C%22pv_count_for_insad%22%3A%7B0%3A-45%2C1%3A1614704504%7D%2C%22insad_views%22%3A%7B0%3A1%2C1%3A1614704504%7D%7D; _cnzz_CV30043604=forum%7Cfid706%7C0; lastvisit=1614653450; lastpath=/post.php",
    "Host": "bbs.nga.cn",
    "sec-ch-ua": '"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"',
    "sec-ch-ua-mobile": "?0",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "same-origin",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
}

paramH = {"tid": 24900465, "authorid": 5254815}
paramL = {"tid": 24900465, "authorid": 150058}
paramG = {"tid": 24900465, "authorid": 2251556}
paramFRED = {"tid": 24900465, "authorid": 34795211}
paramHTG = {"tid": 24900465, "authorid": 60259365}
paramYO = {"tid": 24900465, "authorid": 38574993}
paramLR = {"tid": 24900465, "authorid": 60359242}
paramH_XIN = {"tid": 24900465, "authorid": 614111}
paramQQ = {"tid": 24900465, "authorid": 9439143}


paramN = {"tid": 24913158, "authorid": 4627122}
paramS = {"tid": 24913158, "authorid": 37876}
paramPCG = {"tid": 24913158, "authorid": 43299140}
paramNL = {"tid": 24913158, "authorid": 150058}
paramN_QQ = {"tid": 24913158, "authorid": 11510983}


paramXIN = {"tid": 25668859, "authorid": 614111}

paramLNN = {"tid": 24914654, "authorid": 62458856}

paramHSZ = {"tid": 25909019, "authorid": 62719274}

url1 = "https://bbs.nga.cn/"
# end = "read.php?tid=%s&authorid=%s&page=%s"

def get_page(session, url, header):
    resp = session.get(url=url, headers=header)
    if resp.status_code != 200:
        headers = {
            'Content-Type': 'application/json'
        }
        ding_url1 = "https://oapi.dingtalk.com/robot/send?access_token=354c22f482d29583bdf4636438cda158b8acd5686f42b0929277d933dd95dd49"
        mg = {
            "msgtype": "markdown",
            "markdown": {
                "title":"发言",
                "text": "#### 禾佬发言 \n> cookies 失效了\n"
            },
            "at": {
                "atMobiles": [],
                "isAtAll": False
            }
        }
        # requests.post(ding_url1, data=json.dumps(mg),headers=headers)
    print(resp.status_code)
    return resp.text


def parse(html, authorid):
    html = etree.HTML(html)
    html_data = html.xpath('//html/body/div[@id="mmc"]/div[@id="mc"]/div[@id="m_posts"]/div[@id="m_posts_c"]/table/tr/td[@class="c2"]')
    l = []
    ids = []
    for x in html_data:
        ret = {}
        uid = x.xpath('./a/@id')
        ts = x.xpath('./div/span[@title="reply time"]/text()')
        content = x.xpath('./span/span')
        if len(content) != 1:
            continue
        content2 = etree.tostring(content[0], encoding='utf-8')
        ret["_id"] = uid[0] + str(authorid)
        ret["time"] = ts[0]
        ret["content"] = str(content2, encoding='utf-8')
        ret["send"] = 1
        l.append(ret)
        ids.append(uid[0])
        # print(type(content2))
        # print(uid)
        # print(ts)
        # print(str(content2, encoding='utf-8'))
    
    return l, ids

def get_set_page(html, num):
    html = etree.HTML(html)
    page = html.xpath('//html/body/div[@id="mmc"]/div[@id="mc"]/div[@id="m_pbtnbtm"]/div/div[@class="left"]/a[@title="下一页"]/@href')
    if len(page) == 1:
        page = page[0]
        num = int(page.split("page=")[-1]) - 1
        return True, page, num
    else:
        page = html.xpath('//html/body/div[@id="mmc"]/div[@id="mc"]/div[@id="m_pbtnbtm"]/div/div[@class="left"]/a[@title="上一页"]/@href')
        if len(page) == 1:
            page = page[0]
            num = int(page.split("page=")[-1]) + 1
        return False, page, num


def start(db, url1, end, p, page_num, authorid):
    session = requests.session()
    page1 = get_page(session, url1 + end % (p, page_num), header)
    flag, next_url, num = get_set_page(page1, page_num)
    ret, ids = parse(page1, authorid)
    for i in ret:
        info = db.update(i, {"$setOnInsert": i}, upsert=True)
    # db.insert_many(ret)
    
    while flag:
        time.sleep(1)
        page1 = get_page(session, url1 + next_url, header)
        flag, next_url, num = get_set_page(page1, page_num)
        ret, ids = parse(page1, authorid)
        for i in ret:
            db.update(i, {"$setOnInsert": i}, upsert=True)
    return num

def mongo():
    myclient = pymongo.MongoClient("mongodb://115.159.72.159:7891/")
    db = myclient.admin
    # 认证
    db.authenticate('adminkpmg', 'kpmg123')
    mydb = myclient["spider"]
    mycol = mydb["msg"]
    return mycol


# mdb = mongo()
# numH = start(mdb, url1, end, paramH["authorid"], 1)
# numL = start(mdb, url1, end, paramL["authorid"], 1)
# with open('num.json', 'w') as f:
#     json.dump({"h": numH, "l": numL}, f)



def consist_spider():
    d = {"h": 1, "l": 1, "g": 1, "n": 1, "s": 1, "htg": 1, "pcg": 1, "fred": 1, "nl": 1, "lr": 1, "yo": 1, "xin": 1, "h_xin": 1, "qq": 1, "lnn": 1, "hsz": 1, "n_qq": 1}
    with open('/home/yk/kapp/num.json', 'r') as f:
        d= json.load(f)
    mdb = mongo()
    end1 = "read.php?tid=%s&authorid=%s&page=%s"
    numH, msg_list1 = msg(mdb, url1, end1, paramH["tid"], paramH["authorid"], d["h"])
    numL, msg_list2 = msg(mdb, url1, end1, paramL["tid"], paramL["authorid"], d["l"])
    numG, msg_list3 = msg(mdb, url1, end1, paramG["tid"], paramG["authorid"], d["g"])
    numFRED, msg_list31 = msg(mdb, url1, end1, paramFRED["tid"], paramFRED["authorid"], d["fred"])
    numHTG, msg_list32 = msg(mdb, url1, end1, paramHTG["tid"], paramHTG["authorid"], d["htg"])
    numYO, msg_list33 = msg(mdb, url1, end1, paramYO["tid"], paramYO["authorid"], d["yo"])
    numLR, msg_list34 = msg(mdb, url1, end1, paramLR["tid"], paramLR["authorid"], d["lr"])
    num_H_XIN, msg_list35 = msg(mdb, url1, end1, paramH_XIN["tid"], paramH_XIN["authorid"], d["h_xin"])
    numQQ, msg_list36 = msg(mdb, url1, end1, paramQQ["tid"], paramQQ["authorid"], d["qq"])


    end2 = "read.php?tid=%s&authorid=%s&page=%s"
    numN, msg_list4 = msg(mdb, url1, end2, paramN["tid"], paramN["authorid"], d["n"])
    numS, msg_list5 = msg(mdb, url1, end2, paramS["tid"], paramS["authorid"], d["s"])
    numPCG, msg_list6 = msg(mdb, url1, end2, paramPCG["tid"], paramPCG["authorid"], d["pcg"])
    numNL, msg_list7 = msg(mdb, url1, end2, paramNL["tid"], paramNL["authorid"], d["nl"])
    numN_QQ, msg_list9 = msg(mdb, url1, end2, paramN_QQ["tid"], paramN_QQ["authorid"], d["n_qq"])

    end3 = "read.php?tid=%s&authorid=%s&page=%s"
    numXIN, msg_list8 = msg(mdb, url1, end3, paramXIN["tid"], paramXIN["authorid"], d["xin"])

    end4 = "read.php?tid=%s&authorid=%s&page=%s"
    numLNN, msg_list8 = msg(mdb, url1, end4, paramLNN["tid"], paramLNN["authorid"], d["lnn"])

    end5 = "read.php?tid=%s&authorid=%s&page=%s"
    numHSZ, msg_list8 = msg(mdb, url1, end5, paramHSZ["tid"], paramHSZ["authorid"], d["hsz"])



    print({"h": numH, "l": numL, 'g': numG, 'n': numN, 's': numS, 'htg': numHTG,'pcg': numPCG, "fred": numFRED, "lr": numLR, "xin": numXIN, "h_xin": num_H_XIN, "yo": numYO, "qq": numQQ,  "lnn": numLNN, "hsz": numHSZ, "nl": numNL})
    with open('/home/yk/kapp/num.json', 'w') as f:
        json.dump({"h": numH, "l": numL, 'g': numG, 'n': numN, 's': numS, 'htg': numHTG,'pcg': numPCG, "fred": numFRED, "lr": numLR, "xin": numXIN, "h_xin": num_H_XIN, "yo": numYO, "qq": numQQ,  "lnn": numLNN, "hsz": numHSZ, "nl": numNL, "n_qq": numN_QQ}, f)
    return msg_list1, msg_list2, msg_list3, msg_list31, msg_list32, msg_list34, msg_list4, msg_list5, msg_list6, msg_list8


def msg(db, url1, end, tid, authorid, page_num):
    session = requests.session()
    print(url1 + end % (tid, authorid, page_num))
    page1 = get_page(session, url1 + end % (tid, authorid, page_num), header)
    msg_list = []
    flag, next_url, num = get_set_page(page1, page_num)
    ret, ids = parse(page1, authorid)
    for i in ret:
        try:
            info = db.update(i, {"$setOnInsert": i}, upsert=True)
            if not info["updatedExisting"]:
                pass
                # msg_list.append(i)
        except Exception as e:
            
            p = re.search('"(.*?)"', str(e))
            if p:
                name = p.groups()[0]
                db.delete_many({"_id": name})
            info = db.update(i, {"$setOnInsert": i}, upsert=True)
            if not info["updatedExisting"]:
                # msg_list.append(i)
                pass

    # db.insert_many(ret)
    
    while flag:
        time.sleep(1)
        page1 = get_page(session, url1 + next_url, header)
        flag, next_url, num = get_set_page(page1, page_num)
        ret, ids = parse(page1, authorid)
        for i in ret:
            info = db.update(i, {"$setOnInsert": i}, upsert=True)
            if not info["updatedExisting"]:
                pass
                # msg_list.append(i)
    print(authorid, num)
    return num, msg_list

def mission():
    consist_spider()
    
mission()









